export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
simu_gpu=32
export HYDRA_FULL_ERROR=1
#dataname=ximalaya_redian_2T
dataname=ximalaya_redian_2T_chat_1000
#checkpoint=/home/work_nfs15/asr_data/ckpt/origin_chinese_hubert/chinese_hubert_large.pt
checkpoint=/home/node27_tmpdata/xlgeng/pachong_10W_data/fairseq_data/hubert_feat/chat_1000/hubert_large_iter3_32gpu_1/checkpoint_last.pt
checkpoint=/home/node27_tmpdata/xlgeng/pachong_10W_data/fairseq_data/hubert_feat/ximalaya_redian_2T_chat_1000/hubert_large_iter3_32gpu_1/checkpoint_45_250000.pt
exp_dir=/home/node27_tmpdata/xlgeng/pachong_10W_data/fairseq_data/hubert_feat/$dataname
data_dir=/home/node27_tmpdata/xlgeng/pachong_10W_data/fairseq_data/manifest/$dataname
exp_name=hubert_large_by_xlgeng
all_data=( "train" )
extract_layer=9            # which layer to extract hiddens features
feat=${exp_name}_extract_layer_${extract_layer}
feat_dir=${exp_dir}/feature/${feat}
nj=40
num_gpu=8
num_node=1
km_dir=${exp_dir}/k-means/${feat}
world_size=8
update_freq=4

label_name=["km"]
label_rate=50
model_size=large
conf_name=hubert_large_librivox.yaml
exp_name=hubert_${model_size}_iter3_${simu_gpu}gpu_1
output_dir=${exp_dir}/${exp_name}
conf_dir=config/pretrain/
mkdir -p $output_dir
echo "stage 15: Train the second iteration started @ `date`"
python ../../fairseq_cli/hydra_train.py --config-dir ${conf_dir}\
    --config-name ${conf_name}\
    hydra.run.dir=${output_dir}                                 \
    task.data=${data_dir}                                        \
    task.labels=${label_name}                                   \
    task.label_dir=${km_dir}                                    \
    model.label_rate=${label_rate}                              \
    common.tensorboard_logdir=${output_dir}/tblog               \
    checkpoint.save_dir=${output_dir}                           \
    checkpoint.restore_file=$checkpoint   \
    checkpoint.reset_optimizer=true \
    checkpoint.reset_dataloader=true \
    checkpoint.reset_meters=true \
    distributed_training.distributed_world_size=${world_size}   \
    +optimization.update_freq="[${update_freq}]"
